/*************************************************************************
 *                                                                       *
 * This file is part of the DogosFrame distribution.  For full copyright *
 * information, see COPYRIGHT and COPYING.LESSER.                        *
 *                                                                       *
 * Author:           sqbang                                              *
 * Email:            sqbang@ustc.edu                                     *
 *                                                                       *
 *************************************************************************/

#ifndef DOGOS_include_COMMON_para_DOGOS_MPI_h
#define DOGOS_include_COMMON_para_DOGOS_MPI_h

#include "DogosConfig.h"

#ifdef DOGOS_HAVE_MPI
#  include <mpi.h>
#else

#undef MPI_COMM_WORLD
#undef MPI_COMM_NULL
#undef MPI_SUCCESS
#undef MPI_CONGRUENT
#undef MPI_IDENT
#undef MPI_REQUEST_NULL
#undef MPI_ERR_IN_STATUS
#undef MPI_UNDEFINED
#undef MPI_ANY_SOURCE
#undef MPI_ANY_TAG
#undef MPI_BYTE
#undef MPI_CHAR
#undef MPI_DOUBLE
#undef MPI_FLOAT
#undef MPI_INT
#undef MPI_LONG
#undef MPI_UNSIGNED_LONG
#undef MPI_C_DOUBLE_COMPLEX
#undef MPI_2INT
#undef MPI_DOUBLE_INT
#undef MPI_FLOAT_INT
#undef MPI_MIN
#undef MPI_MINLOC
#undef MPI_MAX
#undef MPI_MAXLOC
#undef MPI_SUM
#undef MPI_TAG_UB

/**
* @brief Enumeration to define MPI constants when compiling without MPI.
*
* These are defined in the global namespace, and it does not matter
* what values these take because they are not used.  (They are just place
* holders to let code compile without MPI without adding excessive
* preprocessor guards to the code.)
*
* This is not a complete set.  Developers should add as needed to extend
* DOGOS_MPI's functionality.
*/
namespace DOGOS
{

enum
{
    MPI_COMM_WORLD,
    MPI_COMM_NULL,
    // Special values:
    MPI_SUCCESS = 0,
    MPI_CONGRUENT,
    MPI_REQUEST_NULL,
    MPI_ERR_IN_STATUS,
    MPI_UNDEFINED,
    MPI_ANY_SOURCE,
    MPI_ANY_TAG,
    // Data types:
    MPI_BYTE,
    MPI_CHAR,
    MPI_DOUBLE,
    MPI_FLOAT,
    MPI_INT,
    MPI_DOUBLE_COMPLEX,
    MPI_2INT,
    MPI_DOUBLE_INT,
    MPI_FLOAT_INT,
    // Operations:
    MPI_MIN,
    MPI_MINLOC,
    MPI_MAX,
    MPI_MAXLOC,
    MPI_SUM,
    // Attributes:
    MPI_TAG_UB
};

} //namespace DOGOS

#endif //define DOGOS_HAVE_MPI

/**
 * @brief Provides C++ wrapper around MPI routines.
 *
 * DOGOS_MPI provides a single point of access in DOGOS for using MPI
 * function calls and for the run-time decision whether to use MPI.
 * The purpose of having a single point is to facilitate writing code
 * that is not cluttered by the repetitive logic absorbed into this class.
 * Codes accessing MPI through this class should work whether DOGOS is
 * configured with or without MPI and whether MPI is enabled at run time.
 *
 * This class provides two sets of interfaces:
 *
 * -# Methods closely matching the MPI interfaces associated with a
 * communicator.  The communicator is set during object construction
 * and is removed from the argument list of the MPI-like interfaces.
 *
 * -# Static methods exactly matching the MPI interfaces not associated
 * with any specific communicator.
 *
 * If DOGOS has MPI enabled, the MPI wrappers in this class delegate to
 * the real MPI methods.  Otherwise, they are no-ops and will throw an
 * assertion if used, except for those specifically identified as able to
 * work without MPI enabled.  The purpose of these interfaces are strictly
 * to check whether MPI was configured into DOGOS and DOGOS is using
 * MPI at run time.  They are not "single-process implementations" of the
 * MPI standard.
 *
 * This class also manages a central DOGOS_MPI object meant for
 * DOGOS's general use.  See get_MPI_world() and init().
 */
class DOGOS_MPI
{
public:

    /**
     * @brief Aliases for MPI data type names.
     *
     * Define aliases for MPI types that can be used whether DOGOS is
     * configured with or without MPI.  Without MPI, these types are "dummies",
     * used to allow the code to be compiled without changes.
     */
#ifdef DOGOS_HAVE_MPI
    typedef MPI_Comm     Comm;
    typedef MPI_Datatype Datatype;
    typedef MPI_Group    Group;
    typedef MPI_Op       Op;
    typedef MPI_Request  Request;
    typedef MPI_Status   Status;
#else
    typedef int          Comm;
    typedef int          Datatype;
    typedef int          Group;
    typedef int          Op;
    typedef int          Request;

    /**
    * @brief Dummy definition of Status to match the MPI standard.
    *
    * Codes are allowed to access the members of this struct, so they must
    * exist for compilation.  Without MPI, we won't actually use them.
    */
    struct Status
    {
        inline Status();
        int MPI_SOURCE;
        int MPI_TAG;
        int MPI_ERROR;
    };
#endif //DOGOS_HAVE_MPI

public:

    /**
     * @brief MPI communicator constants
     */
    static Comm comm_world;
    static Comm comm_null;

    /**
     * @brief Get the primary DOGOS_MPI object owned by DOGOS.
     *
     * This is DOGOS's primary communication object set up when
     * DOGOS_MPI is initialized.  It is used for basic communications
     * that are not associated with another communication object.
     * Various parts of the DOGOS library may create and use (and
     * destroy) communicators that are derived from this object.
     *
     * The use of this object outside of the DOGOS library should be
     * carefully limited to avoid mixing messages.
     *
     * @see init()
     */
    inline static const DOGOS_MPI&
    get_MPI_world();

    /**
     * @brief Get the local process rank from the last time the
     * internal communicator was set.
     */
    inline int
    get_rank() const;

    /**
     * @brief Get the size (number of processes) of the internal
     * communicator the last time it was set.
     */
    inline int
    size() const;

    /**
     * @brief Get the internal communicator.
     */
    inline const Comm&
    get_communicator() const;

    /**
    * @brief Initialize MPI and DOGOS_MPI.
    *
    * This initialization sets up get_MPI_world() to return an
    * DOGOS_MPI object with a communicator duplicated from
    * MPI_COMM_WORLD.
    *
    * Use only one of the three initialization methods.
    *
    * @param[in]  argc  Pointer to parameter from main()
    * @param[in]  argv  Pointer to parameter from main()
    */
    static void
    init (
        int& argc,
        char** argv);

    /**
     * @brief Initialize DOGOS_MPI when MPI is already initialized.
     *
     * This initialization sets up get_MPI_world() to return an
     * DOGOS_MPI object with a communicator duplicated from
     * the one passed in.
     *
     * Use only one of the three initialization methods.
     *
     * @param comm  An MPI Communicator to be duplicated for DOGOS use.
     */
    static void
    init (Comm comm);

    /**
     * @brief Shut down DOGOS_MPI and, if appropriate, shut down MPI.
     *
     * This method frees the MPI communicator duplicated for DOGOS
     * use.  If this class has started MPI with the first init() method,
     * finalize() will also shut down MPI.
     */
    static void
    finalize();

    /**
     * @brief Call MPI_Abort or exit depending on whether running with one
     * or more processes and value set by function above, if called.
     *
     * The default is to call exit(-1) if running with one processor and to
     * call MPI_Abort() otherwise.  This function avoids having to guard abort
     * calls in application code.
     */
    static void
    abort();

private:

    /**
     * @brief Constructor.
     *
     * The given MPI communicator will be used for all communications invoked
     * by this object.
     *
     * Note that the object will NOT automatically free the communicator.
     * To manually free the communicator, use freeCommunicator().
     *
     * If the specified communicator is MPI_COMM_NULL, the rank (see get_rank())
     * and size (see size()) are set to invalid values.  Otherwise, the
     * rank and size are set using the given communicator.  If MPI is enabled
     * but MPI has not been initialized, the rank and size are set to invalid
     * values.  If MPI is not enabled, the rank is set to 0 and size to 1.
     *
     * @param[in] comm
     */
    explicit DOGOS_MPI(
        const Comm& comm);

private:

    /*
     * @brief Internal communicator
     */
    DOGOS_MPI::Comm d_comm;

    /*
     * @brief The local rank.
     */
    int d_rank;

    /*
     * @brief The size of the communicator.
     */
    int d_size;

    /*
     * @brief Whether MPI is initialized.
     */
    static bool s_mpi_is_initialized;

    /*
     * @brief Primary DOGOS_MPI object.
     */
    static DOGOS_MPI s_mpi_world;

private:

    DOGOS_MPI (const DOGOS_MPI&);
    void operator = (const DOGOS_MPI&);
};

#include "COMMON/para/DOGOS_MPI.inl"

#endif //DOGOS_include_COMMON_tbox_DOGOS_MPI_h
